/**
 * @description JSON URL tokenizer
 *
 * Tokens:
 *
 * dot			->	/\./
 *
 * ident		->	/\[string\]/			braced quoted string
 * 				->	/\[raw_braced\]/		braced raw characters
 *				->	/raw/					raw characters
 * 				->	/string/				string
 *
 * Token Subsets:
 *
 * string		->	/\"undquoted\"/
 * 				->	/\'unsquoted\'/
 *
 * undquoted	->	/([^\"]|escaped)+/
 *
 * unsquoted	->	/([^\']|escaped)+/
 *
 * escaped		->	/\\./					typical c style string escape
 * 				->	/\\u[0-9a-f]{1,4}/		utf-8 character escape
 * 				->	/\\x[0-9]{1,2}/			unicode character escape
 *
 * raw			->	/[^operators]/			raw characters except JSON URL operators
 *
 * raw_braced	->	/[^\]]+/				raw characters inside the brace
 *
 * operators	->	/[\.\[\]\"\']/			operators
 *
 */
Jx.categorize('lang.json.url.tokenizer', {

	$extend: 'Jx.code.Tokenizer',

	states: {

		'start': {

			'.': 'dot',

			'[': 'open_brace',

			'\'': 'string_squote',

			'"': 'string_dquote',

			'[*]': 'raw'

		},

		'raw': {

			'[*]': 'raw'

		},

/**
 *	braced
 */
		'open_brace': {

			'\'': 'braced_squote_string',

			'"': 'braced_dquote_string',

			'[*]': 'raw_braced'

		},

		'raw_braced': {

			']': 'braced',

			'[*]': 'raw_braced'

		},

/**
 *	braced single quoted string
 */
		'braced_squote_string': {

			'\'': 'braced_string',

			'\\': 'string_escape',

			'[*]': 'braced_squote_string'

		},

/**
 *	braced double quoted string
 */
		'braced_dquote_string': {

			'\\': 'string_escape',

			'"': 'braced_string',

			'[*]': 'braced_dquote_string'

		},

/**
 *	braced string common
 */
		'braced_string': {

			']': 'braced'

		},

/**
 *	single quoted string
 */

		'string_squote': {

			'\\': 'string_escape',

			'\'': 'string',

			'[*]': 'string_squote'

		},

/**
 *	double quoted string
 */
		'string_dquote': {

			'\\': 'string_escape',

			'"': 'string',

			'[*]': 'string_dquote'

		},

		'string_escape': {

			'u': 'string_utf8_escape',

			'x': 'string_hex_escape',

			'[*]': 'string_default_escape'

		},

		'string_hex_escape': {

			'[*]': 'string_hex_escape'

		},

		'string_utf8_escape': {

			'[*]': 'string_utf8_escape'

		},

		'string_default_escape': {},

/**
 *	end tokens
 */

		'dot': {},

		'braced': {},

		'string': {},

		'ident': {}

	},

	escapeChars: {

		'b': '\b',

		'f': '\f',

		'r': '\r',

		'n': '\n',

		'O': '\O',

		't': '\t',

		'v': '\v'

	},

	stateCallback: {

		'dot': function(chr, input, state, previousState){

			this.appendBuffer(chr);

			return chr;

		},

		'braced': function(chr, input, state, previousState){

			//-- dont append anything
			return 'ident';

		},

		'open_brace': function(chr, input, state, previousState){

			//-- dont append anything

		},

		'braced_squote_string': function(chr, input, state, previousState){

			if (state == previousState){

				this.appendBuffer(chr);

			}

		},

		'braced_dquote_string': function(chr, input, state, previousState){

			if (state == previousState){

				this.appendBuffer(chr);

			}

		},

		'braced_string': function(chr, input, state, previousState){

			//-- dont append anything

		},

		'string_squote': function(chr, input, state, previousState){

			if (state == previousState){

				this.appendBuffer(chr);

			}

		},

		'string_dquote': function(chr, input, state, previousState){

			if (state == previousState){

				this.appendBuffer(chr);

			}

		},

		'string': function(chr, input, state, previousState){

			//-- dont append anything
			return 'ident';

		},

		'string_escape': function(chr, input, state, previousState){

			//-- anchor state
			this.anchorState = previousState;

			delete this.escapedStringBuffer;

		},

		'string_default_escape': function(chr, input, state, previousState){

			this.appendBuffer(

				this.getEscapedChar(chr)

			);

			this.state = this.anchorState;

			delete this.anchorState;

		},

		'string_utf8_escape': function(chr, input, state, previousState){

			return this.hexEscapeCallback( 4, chr, input, state, previousState);

		},

		'string_hex_escape': function(chr, input, state, previousState){

			return this.hexEscapeCallback( 2, chr, input, state, previousState);

		},

		'raw': function(chr, input, state, previousState){

			var returnValue = void(0);

			if (chr in this.states.start){

				this.backtrack(1);

				this.state = 'ident';

			} else {

				this.appendBuffer(chr);

				returnValue = 'ident';

			}


			return 'ident';

		}

	},

	hexRe: /^[0-9a-fA-F]+$/,

	getEscapedChar: function(chr){

		var sets = this.escapeChars;

		return chr in sets ?

			sets[chr]

			:

			chr;

	},

	getUtf8FormattedChar: function(str){

		return String.fromCharCode(

			parseInt(str, 16)

		);

	},

	hexEscapeCallback: function(limit, chr, input, state, previousState){

		//-- initialize buffer
		if (!this.hasOwnProperty('escapedStringBuffer')){

			this.escapedStringBuffer = '';

		//-- apply
		} else {

			var buffer = this.escapedStringBuffer,

				hex = this.hexRe.test(chr),

				ended = !hex || (buffer.length + 1) == limit;

			//-- assign
			if (hex){

				buffer+=chr;

				//-- ended by buffer limit
				if (ended){

					this.appendBuffer(

						this.getUtf8FormattedChar(buffer)

					);

				} else {

					this.escapedStringBuffer = buffer;

				}

			} else {

				this.backtrack(1);

			}


			if (ended){

				delete this.escapedStringBuffer;

				this.state = this.anchorState;

				delete this.anchorState;

			}

		}

	}

});
